<!DOCTYPE html>
<html lang="zh-cn" itemscope itemtype="http://schema.org/WebPage">
<head>
  <meta charset="utf-8">
  <meta http-equiv="X-UA-Compatible" content="IE=edge">
  <title>使用Middleware设置scrapy的User-Agent代理池 - Tonser&#39;s blog</title>
  

<meta name="renderer" content="webkit" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>

<meta name="MobileOptimized" content="width"/>
<meta name="HandheldFriendly" content="true"/>


<meta name="applicable-device" content="pc,mobile">

<meta name="theme-color" content="#f8f5ec" />
<meta name="msapplication-navbutton-color" content="#f8f5ec">
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-status-bar-style" content="#f8f5ec">

<meta name="mobile-web-app-capable" content="yes">

<meta name="author" content="Tonser" />
  <meta name="description" content="使用scrapy爬虫时可能会遇到服务器检测爬虫客户端的headers中user-agent，当被检测到同一个IP地址同一个user-agent有类似爬虫的行为时，服务器会对访问作出一些限制，用来保护正常访问的用户，这无疑对爬虫用户不太友好，本文介绍如何使用user-agent代理池破除这一限制。 " />

  <meta name="keywords" content="Tonser, blog, jane" />






<meta name="generator" content="Hugo 0.47" />


<link rel="canonical" href="https://sundxfansky.club/post/scrapy_crawl_skills/scrapy_user_agent/" />



<link rel="icon" href="/favicon.ico" />











<link rel="stylesheet" href="/sass/jane.min.aaf3394fc81ccb6f6fe3fe458d1f5b61c39f151e44df83baaf1d7b3775a8b98d.css" integrity="sha256-qvM5T8gcy29v4/5FjR9bYcOfFR5E34O6rx17N3WouY0=" media="screen">





<meta property="og:title" content="使用Middleware设置scrapy的User-Agent代理池" />
<meta property="og:description" content="使用scrapy爬虫时可能会遇到服务器检测爬虫客户端的headers中user-agent，当被检测到同一个IP地址同一个user-agent有类似爬虫的行为时，服务器会对访问作出一些限制，用来保护正常访问的用户，这无疑对爬虫用户不太友好，本文介绍如何使用user-agent代理池破除这一限制。
" />
<meta property="og:type" content="article" />
<meta property="og:url" content="https://sundxfansky.club/post/scrapy_crawl_skills/scrapy_user_agent/" /><meta property="article:published_time" content="2018-09-01T00:13:23&#43;08:00"/>
<meta property="article:modified_time" content="2018-09-01T00:13:23&#43;08:00"/>
<meta itemprop="name" content="使用Middleware设置scrapy的User-Agent代理池">
<meta itemprop="description" content="使用scrapy爬虫时可能会遇到服务器检测爬虫客户端的headers中user-agent，当被检测到同一个IP地址同一个user-agent有类似爬虫的行为时，服务器会对访问作出一些限制，用来保护正常访问的用户，这无疑对爬虫用户不太友好，本文介绍如何使用user-agent代理池破除这一限制。
">


<meta itemprop="datePublished" content="2018-09-01T00:13:23&#43;08:00" />
<meta itemprop="dateModified" content="2018-09-01T00:13:23&#43;08:00" />
<meta itemprop="wordCount" content="1592">



<meta itemprop="keywords" content="mysql,scrapy," />
<meta name="twitter:card" content="summary"/>
<meta name="twitter:title" content="使用Middleware设置scrapy的User-Agent代理池"/>
<meta name="twitter:description" content="使用scrapy爬虫时可能会遇到服务器检测爬虫客户端的headers中user-agent，当被检测到同一个IP地址同一个user-agent有类似爬虫的行为时，服务器会对访问作出一些限制，用来保护正常访问的用户，这无疑对爬虫用户不太友好，本文介绍如何使用user-agent代理池破除这一限制。
"/>

<!--[if lte IE 9]>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/classlist/1.1.20170427/classList.min.js"></script>
<![endif]-->

<!--[if lt IE 9]>
  <script src="https://cdn.jsdelivr.net/npm/html5shiv@3.7.3/dist/html5shiv.min.js"></script>
  <script src="https://cdn.jsdelivr.net/npm/respond.js@1.4.2/dest/respond.min.js"></script>
<![endif]-->




</head>
<body>
  <div id="mobile-navbar" class="mobile-navbar">
  <div class="mobile-header-logo">
    <a href="/" class="logo">细嗅蔷薇</a>
  </div>
  <div class="mobile-navbar-icon">
    <span></span>
    <span></span>
    <span></span>
  </div>
</div>
<nav id="mobile-menu" class="mobile-menu slideout-menu">
  <ul class="mobile-menu-list">
    <li class="mobile-menu-item">
        
          
          
            <a class="menu-item-link" href="https://sundxfansky.club/">主页</a>
          
        
      </li><li class="mobile-menu-item">
        
          
          
            <a class="menu-item-link" href="https://sundxfansky.club/post/">文章</a>
          
        
      </li><li class="mobile-menu-item">
        
          
          
            <a class="menu-item-link" href="https://sundxfansky.club/tags/">标签</a>
          
        
      </li><li class="mobile-menu-item">
        
          
          
            <a class="menu-item-link" href="https://sundxfansky.club/categories/">分类</a>
          
        
      </li><li class="mobile-menu-item">
        
          
          
            <a class="menu-item-link" href="https://sundxfansky.club/about/">关于</a>
          
        
      </li>
    
  </ul>
</nav>


  
    






  <link rel="stylesheet" href="/lib/photoswipe/photoswipe.min.css" />
  <link rel="stylesheet" href="/lib/photoswipe/default-skin/default-skin.min.css" />




<div class="pswp" tabindex="-1" role="dialog" aria-hidden="true">

<div class="pswp__bg"></div>

<div class="pswp__scroll-wrap">
    
    <div class="pswp__container">
      <div class="pswp__item"></div>
      <div class="pswp__item"></div>
      <div class="pswp__item"></div>
    </div>
    
    <div class="pswp__ui pswp__ui--hidden">
    <div class="pswp__top-bar">
      
      <div class="pswp__counter"></div>
      <button class="pswp__button pswp__button--close" title="Close (Esc)"></button>
      <button class="pswp__button pswp__button--share" title="Share"></button>
      <button class="pswp__button pswp__button--fs" title="Toggle fullscreen"></button>
      <button class="pswp__button pswp__button--zoom" title="Zoom in/out"></button>
      
      
      <div class="pswp__preloader">
        <div class="pswp__preloader__icn">
          <div class="pswp__preloader__cut">
            <div class="pswp__preloader__donut"></div>
          </div>
        </div>
      </div>
    </div>
    <div class="pswp__share-modal pswp__share-modal--hidden pswp__single-tap">
      <div class="pswp__share-tooltip"></div>
    </div>
    <button class="pswp__button pswp__button--arrow--left" title="Previous (arrow left)">
    </button>
    <button class="pswp__button pswp__button--arrow--right" title="Next (arrow right)">
    </button>
    <div class="pswp__caption">
      <div class="pswp__caption__center"></div>
    </div>
    </div>
    </div>
</div>

  

  

  <header id="header" class="header container">
    <div class="logo-wrapper">
  <a href="/" class="logo">
    
      细嗅蔷薇
    
  </a>
</div>

<nav class="site-navbar">
  <ul id="menu" class="menu">
    
    
        <li class="menu-item">
        
          
          
            <a class="menu-item-link" href="https://sundxfansky.club/">主页</a>
          

        

      </li>
    
        <li class="menu-item">
        
          
          
            <a class="menu-item-link" href="https://sundxfansky.club/post/">文章</a>
          

        

      </li>
    
        <li class="menu-item">
        
          
          
            <a class="menu-item-link" href="https://sundxfansky.club/tags/">标签</a>
          

        

      </li>
    
        <li class="menu-item">
        
          
          
            <a class="menu-item-link" href="https://sundxfansky.club/categories/">分类</a>
          

        

      </li>
    
        <li class="menu-item">
        
          
          
            <a class="menu-item-link" href="https://sundxfansky.club/about/">关于</a>
          

        

      </li>
    
    
  </ul>
</nav>

  </header>

  <div id="mobile-panel">
    <main id="main" class="main bg-llight">
      <div class="content-wrapper">
        <div id="content" class="content container">
          <article class="post bg-white">
    
    <header class="post-header">
      <h1 class="post-title">使用Middleware设置scrapy的User-Agent代理池</h1>
      
      <div class="post-meta">
        <time datetime="2018-09-01" class="post-time">
          2018-09-01
        </time>
        <div class="post-category">
            <a href="https://sundxfansky.club/categories/python/"> python </a>
            <a href="https://sundxfansky.club/categories/%E7%88%AC%E8%99%AB/"> 爬虫 </a>
            <a href="https://sundxfansky.club/categories/%E8%AF%BE%E4%BD%99/"> 课余 </a>
            
          </div>
        <span class="more-meta"> 约 1592 字 </span>
          <span class="more-meta"> 预计阅读 4 分钟 </span>

        
        

        
        
      </div>
    </header>

    
    
<div class="post-toc" id="post-toc">
  <h2 class="post-toc-title">文章目录</h2>
  <div class="post-toc-content">
    <nav id="TableOfContents">
<ul>
<li>
<ul>
<li><a href="#downloadermiddleware-下载器中间件-的使用">DownloaderMiddleware（下载器中间件） 的使用</a></li>
<li><a href="#自己定义middleware">自己定义middleware</a></li>
</ul></li>
</ul>
</nav>
  </div>
</div>

    
    <div class="post-content">
      <p>使用scrapy爬虫时可能会遇到服务器检测爬虫客户端的headers中user-agent，当被检测到同一个IP地址同一个user-agent有类似爬虫的行为时，服务器会对访问作出一些限制，用来保护正常访问的用户，这无疑对爬虫用户不太友好，本文介绍如何使用user-agent代理池破除这一限制。
</p>

<h2 id="downloadermiddleware-下载器中间件-的使用">DownloaderMiddleware（下载器中间件） 的使用</h2>

<p>首先先了解一下scrapy的框架，如图：
<img src="/image/scrapy_architecture_02.png" alt="架构图" />
关于详细的解释后面会专门写一个文章来写，在图中蓝色的方块就是middleware，从图中可以看出有两个middleware，分别是spider和Downloader两个，对于Downloaded的middleware，也就是对scrapy的engine发出的request请求，做一个预处理从而发给服务器去请求html。有关更详细的关于DownloaderMiddleware的描述可以<a href="https://docs.scrapy.org/en/latest/topics/downloader-middleware.html">点击官网</a>查看。</p>

<blockquote>
<p>下载器中间件是介于Scrapy的request/response处理的钩子框架。 是用于全局修改Scrapy request和response的一个轻量、底层的系统。</p>
</blockquote>

<p>要想激活自己的下载器中间件设置需要在<code>settings.py</code>文件中设置(取消注释)：</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre class="chroma"><code class="language-python" data-lang="python"><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span></code></pre></td>
<td class="lntd">
<pre class="chroma"><code class="language-python" data-lang="python"><span class="n">DOWNLOADER_MIDDLEWARES</span> <span class="o">=</span> <span class="p">{</span>
    <span class="s1">&#39;myproject.middlewares.CustomDownloaderMiddleware&#39;</span><span class="p">:</span> <span class="mi">543</span><span class="p">,</span>
<span class="p">}</span></code></pre></td></tr></table>
</div>
</div>
<p>留意官网的说明：</p>

<blockquote>
<p>The <code>DOWNLOADER_MIDDLEWARES</code> setting is merged with the <code>DOWNLOADER_MIDDLEWARES_BASE</code> setting defined in Scrapy (and not meant to be overridden) and then sorted by order to get the final sorted list of enabled middlewares: the first middleware is the one closer to the engine and the last is the one closer to the downloader. In other words, the <code>process_request()</code> method of each middleware will be invoked in increasing middleware order (100, 200, 300, …) and the <code>process_response()</code> method of each middleware will be invoked in decreasing order.</p>

<p>To decide which order to assign to your middleware see the <code>DOWNLOADER_MIDDLEWARES_BASE</code> setting and pick a value according to where you want to insert the middleware. The order does matter because each middleware performs a different action and your middleware could depend on some previous (or subsequent) middleware being applied.</p>

<p>If you want to disable a built-in middleware (the ones defined in <code>DOWNLOADER_MIDDLEWARES_BASE</code> and enabled by default) you must define it in your project’s <code>DOWNLOADER_MIDDLEWARES</code> setting and assign None as its value. For example, if you want to disable the user-agent middleware:</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre class="chroma"><code class="language-python" data-lang="python"><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span><span class="lnt">4
</span></code></pre></td>
<td class="lntd">
<pre class="chroma"><code class="language-python" data-lang="python"><span class="n">DOWNLOADER_MIDDLEWARES</span> <span class="o">=</span> <span class="p">{</span>
    <span class="s1">&#39;myproject.middlewares.CustomDownloaderMiddleware&#39;</span><span class="p">:</span> <span class="mi">543</span><span class="p">,</span>
    <span class="s1">&#39;scrapy.downloadermiddlewares.useragent.UserAgentMiddleware&#39;</span><span class="p">:</span> <span class="bp">None</span><span class="p">,</span>
<span class="p">}</span></code></pre></td></tr></table>
</div>
</div></blockquote>

<p>如果上述的<code>'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,</code>命令没有修改的话，很有可能在自己修改了user-agent之后又被默认的middleware改回去，失去了修改的作用。</p>

<h2 id="自己定义middleware">自己定义middleware</h2>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre class="chroma"><code class="language-python" data-lang="python"><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span></code></pre></td>
<td class="lntd">
<pre class="chroma"><code class="language-python" data-lang="python"><span class="kn">from</span> <span class="nn">fake_useragent</span> <span class="kn">import</span> <span class="n">UserAgent</span>

<span class="k">class</span> <span class="nc">RandomUserAgentMiddleware</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">crawler</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">RandomUserAgentMiddleware</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">ua</span> <span class="o">=</span> <span class="n">UserAgent</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">ua_type</span> <span class="o">=</span> <span class="n">crawler</span><span class="o">.</span><span class="n">settings</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&#34;USER_AGENT_TYPE&#34;</span><span class="p">,</span> <span class="s2">&#34;random&#34;</span><span class="p">)</span>

    <span class="nd">@classmethod</span>
    <span class="k">def</span> <span class="nf">from_crawler</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">crawler</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">cls</span><span class="p">(</span><span class="n">crawler</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">process_request</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">,</span> <span class="n">spider</span><span class="p">):</span>
        <span class="k">def</span> <span class="nf">get_ua_type</span><span class="p">():</span>
            <span class="k">return</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">ua</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">ua_type</span><span class="p">)</span>
        <span class="n">request</span><span class="o">.</span><span class="n">headers</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="sa">b</span><span class="s1">&#39;User-Agent&#39;</span><span class="p">,</span> <span class="n">get_ua_type</span><span class="p">())</span>
        <span class="k">pass</span></code></pre></td></tr></table>
</div>
</div>
<p>使用了python3的<a href="https://github.com/hellysmile/fake-useragent">fake_useragent</a>库,在终端下输入<code>pip install fake-userage</code>安装，按照作者的指示：</p>

<blockquote>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre class="chroma"><code class="language-python" data-lang="python"><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span><span class="lnt">19
</span><span class="lnt">20
</span><span class="lnt">21
</span><span class="lnt">22
</span><span class="lnt">23
</span><span class="lnt">24
</span><span class="lnt">25
</span></code></pre></td>
<td class="lntd">
<pre class="chroma"><code class="language-python" data-lang="python"><span class="kn">from</span> <span class="nn">fake_useragent</span> <span class="kn">import</span> <span class="n">UserAgent</span>
<span class="n">ua</span> <span class="o">=</span> <span class="n">UserAgent</span><span class="p">()</span>

<span class="n">ua</span><span class="o">.</span><span class="n">ie</span>
<span class="c1"># Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US);</span>
<span class="n">ua</span><span class="o">.</span><span class="n">msie</span>
<span class="c1"># Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0)&#39;</span>
<span class="n">ua</span><span class="p">[</span><span class="s1">&#39;Internet Explorer&#39;</span><span class="p">]</span>
<span class="c1"># Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.2; SV1; .NET CLR 3.3.69573; WOW64; en-US)</span>
<span class="n">ua</span><span class="o">.</span><span class="n">opera</span>
<span class="c1"># Opera/9.80 (X11; Linux i686; U; ru) Presto/2.8.131 Version/11.11</span>
<span class="n">ua</span><span class="o">.</span><span class="n">chrome</span>
<span class="c1"># Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.2 (KHTML, like Gecko) Chrome/22.0.1216.0 Safari/537.2&#39;</span>
<span class="n">ua</span><span class="o">.</span><span class="n">google</span>
<span class="c1"># Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_4) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13</span>
<span class="n">ua</span><span class="p">[</span><span class="s1">&#39;google chrome&#39;</span><span class="p">]</span>
<span class="c1"># Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11</span>
<span class="n">ua</span><span class="o">.</span><span class="n">firefox</span>
<span class="c1"># Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:16.0.1) Gecko/20121011 Firefox/16.0.1</span>
<span class="n">ua</span><span class="o">.</span><span class="n">ff</span>
<span class="c1"># Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:15.0) Gecko/20100101 Firefox/15.0.1</span>
<span class="n">ua</span><span class="o">.</span><span class="n">safari</span>
<span class="c1"># Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25</span>
<span class="c1"># and the best one, random via real world browser usage statistic</span>
<span class="n">ua</span><span class="o">.</span><span class="n">random</span></code></pre></td></tr></table>
</div>
</div></blockquote>

<p>查看源码后为<a href="http://d2g6u4gh6d9rq0.cloudfront.net/browsers/fake_useragent_0.1.10.json">json文件</a>,作者自己维护，后期可以考虑自己将其下载下来到数据库中，从本地读取，但是仍推荐使用云端方法，因为有原作者在维护</p>

<p>可以在<code>settings.py</code>中设置<code>USER_AGENT_TYPE</code>类型确定，对于<code>getattr()</code>函数，可以参考官方文档的解读：</p>

<blockquote>
<p>getattr(object, name[, default])</p>

<p>Return the value of the named attribute of object. name must be a string. If the string is the name of one of the object’s attributes, the result is the value of that attribute. <strong>For example, getattr(x, &lsquo;foobar&rsquo;) is equivalent to x.foobar.</strong> If the named attribute does not exist, default is returned if provided, otherwise AttributeError is raised.</p>
</blockquote>

<p>而后<code>settings.py</code>中设置为：</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre class="chroma"><code class="language-python" data-lang="python"><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span><span class="lnt">4
</span></code></pre></td>
<td class="lntd">
<pre class="chroma"><code class="language-python" data-lang="python"><span class="n">DOWNLOADER_MIDDLEWARES</span> <span class="o">=</span> <span class="p">{</span>
    <span class="s1">&#39;addyourpath.middlewares.RandomUserAgentMiddleware&#39;</span><span class="p">:</span> <span class="mi">543</span><span class="p">,</span>
    <span class="s1">&#39;scrapy.downloadermiddlewares.useragent.UserAgentMiddleware&#39;</span><span class="p">:</span> <span class="bp">None</span><span class="p">,</span>
<span class="p">}</span></code></pre></td></tr></table>
</div>
</div>
<p>最后就完成了user-agent代理池的设置
关于<code>settings.py</code>文件又一个小技巧就是，当一个爬虫项目设置多个爬虫网站的时候，可以在spider目录下自己写爬虫代码的项目文件中，如<code>news.py</code>中，在自己的类中加入</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre class="chroma"><code class="language-python" data-lang="python"><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span><span class="lnt">4
</span></code></pre></td>
<td class="lntd">
<pre class="chroma"><code class="language-python" data-lang="python"><span class="n">custom_settings</span> <span class="o">=</span> <span class="p">{</span>
    <span class="c1">#仅作演示</span>
    <span class="s2">&#34;COOKIES_ENABLED&#34;</span><span class="p">:</span> <span class="bp">True</span>
<span class="p">}</span></code></pre></td></tr></table>
</div>
</div>
<p>对应源码可以查看<code>scrapy/spiders/__init__.py</code>中，找到<code>custom_settings</code>。
感谢众多大神的开源项目的帮助。</p>
    </div>

    
    
<div class="post-copyright">
  <p class="copyright-item">
    <span class="item-title">文章作者</span>
    <span class="item-content">Tonser</span>
  </p>
  <p class="copyright-item">
    <span class="item-title">上次更新</span>
    <span class="item-content">2018-09-01</span>
  </p>
  
  <p class="copyright-item">
    <span class="item-title">许可协议</span>
    <span class="item-content"><a rel="license noopener" href="https://creativecommons.org/licenses/by-nc-nd/4.0/" target="_blank">CC BY-NC-ND 4.0</a></span>
  </p>
</div>


    
    

    <footer class="post-footer">
      <div class="post-tags">
          <a href="https://sundxfansky.club/tags/mysql/">mysql</a>
          <a href="https://sundxfansky.club/tags/scrapy/">scrapy</a>
          
        </div>

      
      <nav class="post-nav">
        
          <a class="prev" href="/post/scrapy_crawl_skills/virturalenv_skills/">
            
            <i class="iconfont">
              <svg  class="icon" viewBox="0 0 1024 1024" version="1.1"
  xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
  width="18" height="18">
  <path d="M691.908486 949.511495l75.369571-89.491197c10.963703-12.998035 10.285251-32.864502-1.499144-44.378743L479.499795 515.267417 757.434875 204.940602c11.338233-12.190647 11.035334-32.285311-0.638543-44.850487l-80.46666-86.564541c-11.680017-12.583596-30.356378-12.893658-41.662889-0.716314L257.233596 494.235404c-11.332093 12.183484-11.041474 32.266891 0.657986 44.844348l80.46666 86.564541c1.772366 1.910513 3.706415 3.533476 5.750981 4.877077l306.620399 321.703933C662.505829 963.726242 680.945807 962.528973 691.908486 949.511495z"></path>
</svg>

            </i>
            <span class="prev-text nav-default">Mac下使用virtualenv搭建python的虚拟环境</span>
            <span class="prev-text nav-mobile">上一篇</span>
          </a>
        
          <a class="next" href="/post/scrapy_crawl_skills/scrapy_data_to_mysql/">
            <span class="next-text nav-default">Scrapy爬取数据保存mysql的方法</span>
            <span class="prev-text nav-mobile">下一篇</span>
            
            <i class="iconfont">
              <svg class="icon" viewBox="0 0 1024 1024" version="1.1"
  xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
  width="18" height="18">
  <path d="M332.091514 74.487481l-75.369571 89.491197c-10.963703 12.998035-10.285251 32.864502 1.499144 44.378743l286.278095 300.375162L266.565125 819.058374c-11.338233 12.190647-11.035334 32.285311 0.638543 44.850487l80.46666 86.564541c11.680017 12.583596 30.356378 12.893658 41.662889 0.716314l377.434212-421.426145c11.332093-12.183484 11.041474-32.266891-0.657986-44.844348l-80.46666-86.564541c-1.772366-1.910513-3.706415-3.533476-5.750981-4.877077L373.270379 71.774697C361.493148 60.273758 343.054193 61.470003 332.091514 74.487481z"></path>
</svg>

            </i>
          </a>
      </nav>
    </footer>
  </article>

  
  

  
  

  
  
    <div class="post bg-white">
      <script src="https://utteranc.es/client.js"
            repo= "sundxfansky/comments_blog"
            issue-term="pathname"
            crossorigin="anonymous"
            async>
      </script>
    </div>
  

  

  

  
  
    



        </div>
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="social-links">
  
  
    <a href="mailto:sundxfansky@sjtu.edu.cn" rel="me noopener" class="iconfont"
      title="email" >
      <svg class="icon" viewBox="0 0 1451 1024" version="1.1"
  xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
  width="36" height="36">
  <path d="M664.781909 681.472759 0 97.881301C0 3.997201 71.046997 0 71.046997 0L474.477909 0 961.649408 0 1361.641813 0C1361.641813 0 1432.688811 3.997201 1432.688811 97.881301L771.345323 681.472759C771.345323 681.472759 764.482731 685.154773 753.594283 688.65053L753.594283 688.664858C741.602731 693.493018 729.424896 695.068979 718.077952 694.839748 706.731093 695.068979 694.553173 693.493018 682.561621 688.664858L682.561621 688.65053C671.644501 685.140446 664.781909 681.472759 664.781909 681.472759L664.781909 681.472759ZM718.063616 811.603883C693.779541 811.016482 658.879232 802.205449 619.10784 767.734955 542.989056 701.759633 0 212.052267 0 212.052267L0 942.809523C0 942.809523 0 1024 83.726336 1024L682.532949 1024 753.579947 1024 1348.948139 1024C1432.688811 1024 1432.688811 942.809523 1432.688811 942.809523L1432.688811 212.052267C1432.688811 212.052267 893.138176 701.759633 817.019477 767.734955 777.248 802.205449 742.347691 811.03081 718.063616 811.603883L718.063616 811.603883Z"></path>
</svg>

    </a>
  
    <a href="https://github.com/sundxfansky" rel="me noopener" class="iconfont"
      title="github"  target="_blank"
      >
      <svg class="icon" style="" viewBox="0 0 1024 1024" version="1.1"
  xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
  width="36" height="36">
  <path d="M512 12.672c-282.88 0-512 229.248-512 512 0 226.261333 146.688 418.133333 350.08 485.76 25.6 4.821333 34.986667-11.008 34.986667-24.618667 0-12.16-0.426667-44.373333-0.64-87.04-142.421333 30.890667-172.458667-68.693333-172.458667-68.693333C188.672 770.986667 155.008 755.2 155.008 755.2c-46.378667-31.744 3.584-31.104 3.584-31.104 51.413333 3.584 78.421333 52.736 78.421333 52.736 45.653333 78.293333 119.850667 55.68 149.12 42.581333 4.608-33.109333 17.792-55.68 32.426667-68.48-113.706667-12.8-233.216-56.832-233.216-253.013333 0-55.893333 19.84-101.546667 52.693333-137.386667-5.76-12.928-23.04-64.981333 4.48-135.509333 0 0 42.88-13.738667 140.8 52.48 40.96-11.392 84.48-17.024 128-17.28 43.52 0.256 87.04 5.888 128 17.28 97.28-66.218667 140.16-52.48 140.16-52.48 27.52 70.528 10.24 122.581333 5.12 135.509333 32.64 35.84 52.48 81.493333 52.48 137.386667 0 196.693333-119.68 240-233.6 252.586667 17.92 15.36 34.56 46.762667 34.56 94.72 0 68.522667-0.64 123.562667-0.64 140.202666 0 13.44 8.96 29.44 35.2 24.32C877.44 942.592 1024 750.592 1024 524.672c0-282.752-229.248-512-512-512"></path>
</svg>

    </a>


<a href="https://sundxfansky.club/index.xml" rel="noopener alternate" type="application/rss&#43;xml"
    class="iconfont" title="rss" target="_blank">
    <svg class="icon" viewBox="0 0 1024 1024" version="1.1"
  xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
  width="30" height="30">
  <path d="M819.157333 1024C819.157333 574.592 449.408 204.8 0 204.8V0c561.706667 0 1024 462.293333 1024 1024h-204.842667zM140.416 743.04a140.8 140.8 0 0 1 140.501333 140.586667A140.928 140.928 0 0 1 140.074667 1024C62.72 1024 0 961.109333 0 883.626667s62.933333-140.544 140.416-140.586667zM678.784 1024h-199.04c0-263.210667-216.533333-479.786667-479.744-479.786667V345.173333c372.352 0 678.784 306.517333 678.784 678.826667z"></path>
</svg>

  </a>
  
</div>

<div class="copyright">
  <span class="power-by">
    Powered by <a class="hexo-link" href="https://gohugo.io">Hugo</a>
  </span>
  <span class="division">|</span>
  <span class="theme-info">
    Theme - <a class="theme-link" href="https://github.com/xianmin/hugo-theme-jane">Jane</a>
  </span>

  <span class="copyright-year">
    &copy;
    
      2017 -
    2019
    <span class="heart">
      
      <i class="iconfont">
        <svg class="icon" viewBox="0 0 1025 1024" version="1.1"
  xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
  width="14" height="14">
  <path d="M1000.1 247.9c-15.5-37.3-37.6-70.6-65.7-98.9-54.4-54.8-125.8-85-201-85-85.7 0-166 39-221.4 107.4C456.6 103 376.3 64 290.6 64c-75.1 0-146.5 30.4-201.1 85.6-28.2 28.5-50.4 61.9-65.8 99.3-16 38.8-24 79.9-23.6 122.2 0.7 91.7 40.1 177.2 108.1 234.8 3.1 2.6 6 5.1 8.9 7.8 14.9 13.4 58 52.8 112.6 102.7 93.5 85.5 209.9 191.9 257.5 234.2 7 6.1 15.8 9.5 24.9 9.5 9.2 0 18.1-3.4 24.9-9.5 34.5-30.7 105.8-95.9 181.4-165 74.2-67.8 150.9-138 195.8-178.2 69.5-57.9 109.6-144.4 109.9-237.3 0.1-42.5-8-83.6-24-122.2z"
   fill="#8a8a8a"></path>
</svg>

      </i>
    </span><span class="author">
        Tonser
        
      </span></span>

  
  
</div>

    </footer>

    <div class="back-to-top" id="back-to-top">
      <i class="iconfont">
        
        <svg class="icon" viewBox="0 0 1024 1024" version="1.1"
  xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
  width="35" height="35">
  <path d="M510.866688 227.694839 95.449397 629.218702l235.761562 0-2.057869 328.796468 362.40389 0L691.55698 628.188232l241.942331-3.089361L510.866688 227.694839zM63.840492 63.962777l894.052392 0 0 131.813095L63.840492 195.775872 63.840492 63.962777 63.840492 63.962777zM63.840492 63.962777"></path>
</svg>

      </i>
    </div>
  </div>
  
<script type="text/javascript" src="/lib/jquery/jquery-3.2.1.min.js"></script>
  <script type="text/javascript" src="/lib/slideout/slideout-1.0.1.min.js"></script>




<script type="text/javascript" src="/js/main.dee43230127a73d039a734510fa896c89c3c7ce0cf0be0c7a7433f8fd69b76dc.js" integrity="sha256-3uQyMBJ6c9A5pzRRD6iWyJw8fODPC&#43;DHp0M/j9abdtw="></script>






  
    <script type="text/javascript" src="/js/load-photoswipe.js"></script>
    <script type="text/javascript" src="/lib/photoswipe/photoswipe.min.js"></script>
    <script type="text/javascript" src="/lib/photoswipe/photoswipe-ui-default.min.js"></script>
  













</body>
</html>
